const axios = require('axios');

// const fs = require('fs');

// const writeStream = fs.createWriteStream('test.csv');

// // 写入csv首行信息
// writeStream.write('时间,标题,网址 \n');

// const url = 'https://www.qschina.cn/university-rankings/university-subject-rankings/2020/communication-media-studies';
// axios.get(url)
//     .then(function (response) {
//         // console.log(response.data);
//         getDataUrl(response.data);

//     })
//     .catch(function (error) {
//         // handle error
//         console.log(error);
//     });

function getDataUrl(str) {
    const regex = /rank_url.+?(\d+\.txt)/;
    let m, dataUrl;

    if ((m = regex.exec(str)) !== null) {
        dataUrl = 'https://www.qschina.cn/sites/default/files/qs-rankings-data/cn/';
        // console.log(dataUrl + m[1]);

        // https://www.qschina.cn/sites/default/files/qs-rankings-data/cn/2005378.txt
        return dataUrl + m[1];
    }
}

function getDataHistroy() {
    let DataHistroy = [];
    for (let index = 2017; index <= 2020; index++) {
        // https://www.qschina.cn/university-rankings/university-subject-rankings/2020/communication-media-studies
        DataHistroy.push(`https://www.qschina.cn/university-rankings/university-subject-rankings/${index}/communication-media-studies`);

    }
    // console.log(DataHistroy);
    return DataHistroy;
}

function getDataUrlList() {
    let dataHistroy = [];
    dataHistroy = getDataHistroy();
    dataHistroy.forEach(url => {
        // console.log(url);
        axios.get(url)
            .then(function (response) {
                // console.log(response.data);
                // console.log(getDataUrl(response.data));
                //  getDataUrl(response.data);             
                // axios.get(getDataUrl(response.data)).then(function (response) {
                //     console.log(response.data);
                // });
            })
            .catch(function (error) {
                // handle error
                console.log(error);
            });

    });

}

// getDataUrlList();

function getData(url) {

    axios.get(url).then(function (response) {
        console.log(response.data);
    });
}
